#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Date    : 2017-06-29 09:46:24
# @Author  : Your Name (you@example.org)
# @Link    : http://example.org
# @Version : $Id$

import time
import httplib
import urllib2
from bs4 import BeautifulSoup


class SpiderFunc(object):
    '''
    self.response 网站源代码
    self.soup 网站
    self.cityList 所有的字典
    self.pageNum 二级页面下的所有页面总数
    self.itemNum 二级页面下的数据项数目
    self.itemUrl 三级页面跳转地址
    '''

    def __init__(self):
        self.response = ''

    '''下载网站(xf.house.163.com)源代码 '''

    def get_xf_house_page(self, url):
        # url = "http://xf.house.163.com"
        try:
            res = urllib2.urlopen(url).read()
        except urlList.HTTPError as e:
            print e

        self.response = unicode(res, 'GBK').encode('UTF-8')  # GBK = gb2312
        return self.response

    '''获取城市和地址的字典'''

    def getCityList(self):
        cityList = []
        # BeautifulSoup 默认将文档流解析成为unicode编码
        # 设置 from_encoding 保证中文不出现乱码; 指定文档解析方式为html5lib
        soup = BeautifulSoup(self.response, 'html5lib', from_encoding="utf-8")
        # 取出城市名，以及对应的城市的链接地址
        cityname = soup.select('h3 a')[4:]  # 除去前面四个重复的楼盘信息
        for c in cityname:
            tempVal = []  # 临时
            # 获取城市名，将unicode转换成为str的UTF-8编码
            tempVal.append(c.get_text().encode('utf-8'))
            url = c['href'].encode('utf-8')
            # 获取标准的URL地址
            if (len(url.split("/")[5]) <= 0):
                url = url + '0-0-0-0-0-0-0-0-0-1-0-0-0-0-0-1-1-0-0-0-1.html'
            else:
                url = url
            tempVal.append(url)
            cityList.append(tempVal)

        return cityList

    ''' 获取每一个城市对应的页面的总的数量,数据项数,以及链接地址
        self.pageNum 为二级页面遍历总数，
        self.itemNum 当前页面的数据项数
        self.itemUrlList 为当前item对应的所有的链接地址
    '''

    def getSecondPageData(self, url):  # 此处的url地址为二级页面下的url地址（包含了列表项的）
        '''
        res 网站源代码
        soup 转换成为 BeautifulSoup 对象
        a 标签的特定父节点
        href 末页的地址链
        '''
        try:
            res = urllib2.urlopen(url).read()
        except urllib2.URLError as e:
            time.sleep(60*1)
            res = urllib2.urlopen(url).read()
            print e
        except urllib2.HTTPError as e:
            time.sleep(60*1)
            res = urllib2.urlopen(url).read()
            print e
        
        soup = BeautifulSoup(res, 'html5lib', from_encoding='utf-8')

        # 分析该地址链，取出总的页面的数量
        dom = soup.find_all("div", class_="pager_box")[-1]  # 获取当前div
        href = dom.contents[-2]['href']  # 获取 末页 的 href 属性值
        numlist = href.split('-')
        pageNum = numlist[-5]

        # 数据项数目
        itemList = soup.find_all('div', class_="sale_cont")
        itemNum = len(itemList)

        # 链接地址
        itemUrlList = []  # 先将之前存下来的item置空
        for i in itemList:
            itemUrlList.append(i['_link'])

        return int(pageNum), itemNum, itemUrlList

    '''下载指定楼盘的网站源码'''

    def getDetailData(self, url):  # 此处的URL地址为三级页面下的地址
        # 改成HTTP1.0协议
        httplib.HTTPConnection._http_vsn = 10
        httplib.HTTPConnection._http_vsn_str = 'HTTP/1.0'

        # python能够改变变量作用域的代码段是def、class、lamda.
        try:
            req = urllib2.Request(url)
            res = (urllib2.urlopen(req)).read()
        except urllib2.URLError as e:
            print e
            time.sleep(60*1)  # 如果连接错误[URLError: <urlopen error [Errno 10060] >],休息两分钟再试试看吧！
            req = urllib2.Request(url)
            res = (urllib2.urlopen(req)).read()
        except urllib2.HTTPError as e:
            print e
            time.sleep(60*1)  # 如果连接错误[URLError: <urlopen error [Errno 10060] >],休息两分钟再试试看吧！
            req = urllib2.Request(url)
            res = (urllib2.urlopen(req)).read()

        soup = BeautifulSoup(res, 'html5lib', from_encoding='utf-8')

        itemDetailData = []  # 先将详细数据列表置空

        '''如果一旦出现网站可以打开，但是页面为空的情况的就直接返回空的值了'''
        if(soup.find_all('span')):
            # 街区
            distract_ = soup.find_all('span', class_='text_area')
            if(len(distract_) > 0):
                # 名字
                name_ = soup.find_all('h1', class_='big_name')
                name = (name_[0].get_text()).encode("utf-8")
                name = name.replace(" ", "")
                name = name.replace("\t", "")
                name = name.replace("\n", "")
                itemDetailData.append(name)

                distract = (distract_[0].get_text().strip()).encode("utf-8")
                distract = distract.replace(" ", "")
                distract = distract.replace("\t", "")
                distract = distract.replace("\n", "")
                itemDetailData.append(distract)

                # 详细地址
                address_ = soup.find_all('span', 'text3')
                address = address_[-1].get_text().strip().encode("utf-8")
                itemDetailData.append(address)

                # 价格
                price_ = soup.find_all('span', class_='text1')
                price = (price_[0].get_text().encode("utf-8")).strip()
                itemDetailData.append(price)

                # 电话
                telphone_ = soup.find_all('span', class_='tel_text')
                telphone = (telphone_[0].get_text().encode('utf-8')).strip()
                itemDetailData.append(telphone)
            else:
                pass        
            # 其他数据
            # other 包含在ul的dom数据列表
            other = soup.find_all('ul', class_='i_info_list')

            '''将三级页面的em数据down下来'''
            i = 1
            deta_il_temp = []
            while i < len(other[0].contents):  # other[0].contents  是 li 数据项列表
                # (other[0].contents)[i].contents 是 li 下的所有标签 , 所有的em
                liChildList = (other[0].contents)[i].contents
                n = 1
                while n < len(liChildList):
                    deta_il_temp.append(
                        (liChildList[n].get_text()).encode("utf-8"))
                    n = n + 2
                i = i + 2  # 遍历项目下的奇数次项，中间很多制表符，很气人，注意代码规范啊！
            else:
                pass

            for a in deta_il_temp:
                itemDetailData.append(a.strip())  # 清除掉 "\n" "\t"  ""
        else:
            itemDetailData.append('null')

        return itemDetailData  # 获取的详细的数据项列表


''' test test test'''
if __name__ == '__main__':

    timeSleep = 60 * 10  # 每爬取1000行数据休息十分钟

    ISOTIMEFORMAT = '%Y-%m-%d %X'
    currentTime = time.strftime(ISOTIMEFORMAT, time.localtime())

    allDataInfo = []  # 存储所有数据

    print currentTime + "列表初始化完成"

    spf = SpiderFunc()

    print currentTime + "爬虫任务初始化完成"

    spf.get_xf_house_page('http://xf.house.163.com')  # 设定爬取的网站地址

    print currentTime + '网站源代码下载完成'

    list_ = spf.getCityList()  # 一级页面 获取城市楼盘和url

    print currentTime + "城市楼盘列表获取完成"

    # for i in list_:print i[1]

    for cl in list_:

        if ((len(allDataInfo) % 2000) == 0 and len(allDataInfo) > 2000):
            # time.sleep(timeSleep)
            print "------------" + currentTime + "休息十分钟先------------"
        else:
            # 福州楼盘地址默认打不开，选择跳过
            if cl[0] == "福州新楼盘" or cl[0] == "衡水新楼盘":
                continue  # continue 跳出当前循环，break 跳出整个循环

            print currentTime + cl[0] + '任务开始'

            url = cl[1]  # 城市对应的链接地址
            # 获取当前的短链接地址  http://xf.house.163.com/bj/search/
            shortUrl = "http://xf.house.163.com"

            # 第一次进入二级页面的数据，获取当前城市的页面数量 pageNum
            pageNum = (spf.getSecondPageData(url))[0]
            print "当前城市列表有" + str(pageNum) + '页'

            for n in range(1, pageNum + 1):  # 拼接次级页面地址 securl
                tempTuple = []
                securl = url[:-14] + str(n) + '-0-0-0-1.html'

                # 第二次爬取二级页面的数据，爬取页面的项数-itemNum，以及每一项对应的URL地址列表 urlList
                tempTuple = spf.getSecondPageData(securl)
                itemNum = tempTuple[1]
                urlList = tempTuple[2]

                # 遍历url list 并跳转到三级页面下载文件
                i = 1
                for u in urlList:  # 依次爬取url list中的数据
                    tempItemData = []  # 储存每一项数所有的基本信息
                    tempItemData.append(cl[0])
                    thirdPageUrl = shortUrl + u
                    print "正在爬取第" + str(n) + '页第' + str(i) + '项，网址是' + thirdPageUrl.encode("utf-8")
                    details = spf.getDetailData(thirdPageUrl)
                    i = i + 1
                    for d in details:
                        tempItemData.append(d)

                    allDataInfo.append(tempItemData)

                    # 将爬虫结果写到如本地text中
                    with open('data.txt', 'w') as f:
                        c = 1
                        for a in allDataInfo:
                            f.write(str(c) + "\t")
                            c = c + 1
                            for l in a:
                                f.write(l)
                                f.write(",")
                            f.write("\n")

        print currentTime + cl[0] + "结束"

    print '结束'
